In [ ]:
searchterm="big data" #lowecase!
In [ ]:
import cx_Oracle #ensure that OS, InstantClient (Basic, ODBC, SDK) and cx_Oracle are all 64 bit. Install with "pip install cx_Oracle". Add link to InstantClient in Path variable!
import pandas as pd
import re
import networkx as nx
In [ ]:
dsn_tns=cx_Oracle.makedsn('127.0.0.1','6025',service_name='bibliodb01.fiz.karlsruhe')
#open connection:
db=cx_Oracle.connect(<username>, <password>, dsn_tns)
print(db.version)
In [ ]:
#%% functions:
def read_query(connection, query):
cursor = connection.cursor()
try:
cursor.execute( query )
names = [ x[0] for x in cursor.description]
rows = cursor.fetchall()
return pd.DataFrame( rows, columns=names)
finally:
if cursor is not None:
cursor.close()
def add_word_to_dict_and_get_id(word,nodes,nodeid):
if word in nodes.keys():
myid=nodes[word]
else:
G.add_node(nodeid, text=word)
nodes[word]=nodeid
nodeid=nodeid+1
myid=nodes[word]
return myid,nodeid
def add_edge_and_increase_weight(id1,id2,edgeid):
#make searchids in both directions:
edgeid1=str(id1)+'-'+str(id2)
edgeid2=str(id2)+'-'+str(id1)
if edgeid1 in edges.keys():
#add weight +1:
G[id1][id2]['weight']=G[id1][id2]['weight']+1
elif edgeid2 in edges.keys():
G[id2][id1]['weight']=G[id2][id1]['weight']+1
else: #both combinations do not exist yet -> create new:
G.add_edge(id1,id2,weight=1)
edges[edgeid1]=1
edgeid=edgeid+1
return edgeid
In [ ]:
#%% load paper titles from WOSdb:
database="wos12b"
command="""SELECT DISTINCT(ARTICLE_TITLE)
FROM """+database+""".KEYWORDS, """+database+""".ITEMS_KEYWORDS, """+database+""".ITEMS
WHERE
"""+database+""".ITEMS_KEYWORDS.FK_KEYWORDS="""+database+""".KEYWORDS.PK_KEYWORDS
AND """+database+""".ITEMS.PK_ITEMS="""+database+""".ITEMS_KEYWORDS.FK_ITEMS
AND (lower("""+database+""".KEYWORDS.KEYWORD) LIKE '%"""+searchterm+"""%' OR lower(ARTICLE_TITLE) LIKE '%"""+searchterm+"""%')
"""
dfWOS=read_query(db,command)
dfWOS.to_csv("all_big_data_titles_wos.csv", sep=';')
#%% load paper titles from SCOPUSdb:
database="SCOPUS20b"
command="""SELECT DISTINCT(ARTICLE_TITLE)
FROM """+database+""".KEYWORDS, """+database+""".ITEMS_KEYWORDS, """+database+""".ITEMS
WHERE
"""+database+""".ITEMS_KEYWORDS.FK_KEYWORDS="""+database+""".KEYWORDS.PK_KEYWORDS
AND """+database+""".ITEMS.PK_ITEMS="""+database+""".ITEMS_KEYWORDS.FK_ITEMS
AND (lower("""+database+""".KEYWORDS.KEYWORD) LIKE '%"""+searchterm+"""%' OR lower(ARTICLE_TITLE) LIKE '%"""+searchterm+"""%')
"""
dfSCOPUS=read_query(db,command)
dfSCOPUS.to_csv("all_big_data_titles_scopus.csv", sep=';')
#%% join both dataframes:
df=pd.merge(dfWOS,dfSCOPUS,on='ARTICLE_TITLE',how='outer') #130 identical titles
df.to_csv("all_big_data_titles.csv", sep=';')
In [ ]:
stopwords=['case','study','new','s','real','multi','use','2013','age','e','novel','control','level','high','using','challenges','networks','application','applications','approach','based','a', 'able', 'about', 'above', 'according', 'accordingly', 'across', 'actually', 'after', 'afterwards', 'again', 'against', 'ain’t', 'all', 'allow', 'allows', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'an', 'and', 'another', 'any', 'anybody', 'anyhow', 'anyone', 'anything', 'anyway', 'anyways', 'anywhere', 'apart', 'appear', 'appreciate', 'appropriate', 'are', 'aren’t', 'around', 'as', 'aside', 'ask', 'asking', 'associated', 'at', 'available', 'away', 'awfully', 'be', 'became', 'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'believe', 'below', 'beside', 'besides', 'best', 'better', 'between', 'beyond', 'both', 'brief', 'but', 'by', 'c’mon', 'c', 'came', 'can', 'can’t', 'cannot', 'cant', 'cause', 'causes', 'certain', 'certainly', 'changes', 'clearly', 'co', 'com', 'come', 'comes', 'concerning', 'consequently', 'consider', 'considering', 'contain', 'containing', 'contains', 'corresponding', 'could', 'couldn’t', 'course', 'currently', 'definitely', 'described', 'despite', 'did', 'didn’t', 'different', 'do', 'does', 'doesn’t', 'doing', 'don’t', 'done', 'down', 'downwards', 'during', 'each', 'edu', 'eg', 'eight', 'either', 'else', 'elsewhere', 'enough', 'entirely', 'especially', 'et', 'etc', 'even', 'ever', 'every', 'everybody', 'everyone', 'everything', 'everywhere', 'ex', 'exactly', 'example', 'except', 'far', 'few', 'fifth', 'first', 'five', 'followed', 'following', 'follows', 'for', 'former', 'formerly', 'forth', 'four', 'from', 'further', 'furthermore', 'get', 'gets', 'getting', 'given', 'gives', 'go', 'goes', 'going', 'gone', 'got', 'gotten', 'greetings', 'had', 'hadn’t', 'happens', 'hardly', 'has', 'hasn’t', 'have', 'haven’t', 'having', 'he', 'he’s', 'hello', 'help', 'hence', 'her', 'here', 'here’s', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers', 'herself', 'hi', 'him', 'himself', 'his', 'hither', 'hopefully', 'how', 'howbeit', 'however', 'i’d', 'i’ll', 'i’m', 'i’ve', 'ie', 'if', 'ignored', 'immediate', 'in', 'inasmuch', 'inc', 'indeed', 'indicate', 'indicated', 'indicates', 'inner', 'insofar', 'instead', 'into', 'inward', 'is', 'isn’t', 'it', 'it’d', 'it’ll', 'it’s', 'its', 'itself', 'just', 'keep', 'keeps', 'kept', 'know', 'knows', 'known', 'last', 'lately', 'later', 'latter', 'latterly', 'least', 'less', 'lest', 'let', 'let’s', 'like', 'liked', 'likely', 'little', 'look', 'looking', 'looks', 'ltd', 'mainly', 'many', 'may', 'maybe', 'me', 'mean', 'meanwhile', 'merely', 'might', 'more', 'moreover', 'most', 'mostly', 'much', 'must', 'my', 'myself', 'name', 'namely', 'nd', 'near', 'nearly', 'necessary', 'need', 'needs', 'neither', 'never', 'nevertheless', 'new', 'next', 'nine', 'no', 'nobody', 'non', 'none', 'noone', 'nor', 'normally', 'not', 'nothing', 'novel', 'now', 'nowhere', 'obviously', 'of', 'off', 'often', 'oh', 'ok', 'okay', 'old', 'on', 'once', 'one', 'ones', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'ought', 'our', 'ours', 'ourselves', 'out', 'outside', 'over', 'overall', 'own', 'particular', 'particularly', 'per', 'perhaps', 'placed', 'please', 'plus', 'possible', 'presumably', 'probably', 'provides', 'que', 'quite', 'qv', 'rather', 'rd', 're', 'really', 'reasonably', 'regarding', 'regardless', 'regards', 'relatively', 'respectively', 'right', 'said', 'same', 'saw', 'say', 'saying', 'says', 'second', 'secondly', 'see', 'seeing', 'seem', 'seemed', 'seeming', 'seems', 'seen', 'self', 'selves', 'sensible', 'sent', 'serious', 'seriously', 'seven', 'several', 'shall', 'she', 'should', 'shouldn’t', 'since', 'six', 'so', 'some', 'somebody', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhat', 'somewhere', 'soon', 'sorry', 'specified', 'specify', 'specifying', 'still', 'sub', 'such', 'sup', 'sure', 't’s', 'take', 'taken', 'tell', 'tends', 'th', 'than', 'thank', 'thanks', 'thanx', 'that', 'that’s', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'thence', 'there', 'there’s', 'thereafter', 'thereby', 'therefore', 'therein', 'theres', 'thereupon', 'these', 'they', 'they’d', 'they’ll', 'they’re', 'they’ve', 'think', 'third', 'this', 'thorough', 'thoroughly', 'those', 'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together', 'too', 'took', 'toward', 'towards', 'tried', 'tries', 'truly', 'try', 'trying', 'twice', 'two', 'un', 'under', 'unfortunately', 'unless', 'unlikely', 'until', 'unto', 'up', 'upon', 'us', 'use', 'used', 'useful', 'uses', 'using', 'usually', 'value', 'various', 'very', 'via', 'viz', 'vs', 'want', 'wants', 'was', 'wasn’t', 'way', 'we', 'we’d', 'we’ll', 'we’re', 'we’ve', 'welcome', 'well', 'went', 'were', 'weren’t', 'what', 'what’s', 'whatever', 'when', 'whence', 'whenever', 'where', 'where’s', 'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', 'who’s', 'whoever', 'whole', 'whom', 'whose', 'why', 'will', 'willing', 'wish', 'with', 'within', 'without', 'won’t', 'wonder', 'would', 'would', 'wouldn’t', 'yes', 'yet', 'you', 'you’d', 'you’ll', 'you’re', 'you’ve', 'your', 'yours', 'yourself', 'yourselves', 'zero']
In [ ]:
nodes = {}
edges= {}
G=nx.Graph()
nodeid=1
edgeid=1
for title in df['ARTICLE_TITLE']:
#for each title, create and empty list of words/nodes that have been visited
#in this title already to prevent higher edge counts through double visit:
visited_words_in_this_title=[]
wordList = re.sub("[^\w]", " ", title).split()
for word in wordList:
word=word.lower()
if not word in stopwords:
visited_words_in_this_title.append(word)
#before adding node, check if it already exists:
myid,nodeid=add_word_to_dict_and_get_id(word,nodes,nodeid)
#now we know that the present word is in the dict and have the right ID
#loop through all other words in the same title (wordlist):
for word2 in wordList:
word2=word2.lower()
if not word2 in stopwords:
if (word2!=word) & (not word2 in visited_words_in_this_title):
id2,nodeid=add_word_to_dict_and_get_id(word2,nodes,nodeid)
#now both nodes are in the dict and we have both ids
#check if edge already exists:
edgeid=add_edge_and_increase_weight(myid,id2,edgeid)
SG=nx.Graph( [ (u,v,d) for u,v,d in G.edges(data=True) if d
['weight']>1] )
In [ ]:
nx.write_graphml(G,"D:\\Dropbox\\Arbeit\\Keyword-Analysis\\BigData.graphml")
nx.write_graphml(SG,"D:\\Dropbox\\Arbeit\\Keyword-Analysis\\BigData_SG.graphml")